package com.junmeng.region;

import com.google.gson.Gson;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;

/**
 * 从国家统计局网站（http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/）爬取指定年份的区划数据
 * 数据以json格式保存
 * 可以指定抓取的数据范围，比如全中国或指定省份或指定城市等
 * 可以指定抓取的数据层级，比如只抓取到市或区或街道等
 */
public class GenRegionCodeFromWeb {

    //以下值从要抓取的网页分析获得，为元素里的class值（比如<tr class="citytr"><td><a href="44/4401.html">440100000000</a></td><td><a href="44/4401.html">广州市</a></td></tr>）
    public static final String LEVEL_CHINA = "chinatr"; //此项并非网页中拥有，仅仅是为了方便本工程
    public static final String LEVEL_PROVINCE = "provincetr";
    public static final String LEVEL_CITY = "citytr";
    public static final String LEVEL_COUNTY = "countytr";
    public static final String LEVEL_TOWN = "towntr";
    public static final String LEVEL_VILLAGE = "villagetr";

    //<层级字符串，对应的层级数字>
    private static Map<String, Integer> levelMap = new HashMap<>();
    private static List<String> citysWithNextLevelNotCounty = new ArrayList<>();

    private static List<FailedInfo> failedItems = new ArrayList<>();//存放在抓取过程中失败的url

    static {
        levelMap.put(LEVEL_PROVINCE, 1);//表示省,直辖市，自治区,第一级数据
        levelMap.put(LEVEL_CITY, 2);//表示城市,第二级数据
        levelMap.put(LEVEL_COUNTY, 3);//表示区,第三级数据
        levelMap.put(LEVEL_TOWN, 4);//表示街道,第四级数据
        levelMap.put(LEVEL_VILLAGE, 5);//表示居委会,第五级数据
    }

    //这个列表存放的是比较特殊的市，它们是属于LEVEL_CITY，但下一级却跳过了LEVEL_COUNTY，而直接到LEVEL_TOWN
    //由于数据较多，不能一一比对，使用当中发现属于这种情况的城市加入到这里即可
    static {
        citysWithNextLevelNotCounty.add("东莞市");
        citysWithNextLevelNotCounty.add("中山市");
        citysWithNextLevelNotCounty.add("儋州市");
    }

    //保存抓取结果的路径
    public static final String resultStorePath = "./src/main/resources/static";
    //保存异常的url
    public static final String errorLogStorePath = "./src/main/resources/static/error.log";

    //**************************以下值请根据实际情况修改*************************************
    //字段解释：
    //startRegion: 抓取区划的起点,可从国家统计局查看
    //startLevel: startRegion对应的级别，见LEVEL_开头的
    //startCode: startRegion对应的区划编码，可从国家统计局查看
    //startUrl: startRegion在国家统计局中所对应的网页：http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html
    //TARGET_LEVEL: 表示要抓取到哪个层级的数据，比如startRegion为中国，TARGET_LEVEL为LEVEL_CITY，则表示抓取到全国的城市

    //抓取全国的城市
//    public static final String startRegion = "中国";
//    public static final String startLevel = LEVEL_CHINA;
//    public static final String startCode = "000000000000";
//    public static final String startUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html";
//    public static String TARGET_LEVEL = LEVEL_CITY;

    //抓取广东省所有区的数据
//    public static final String startRegion = "广东省";
//    public static final String startLevel = LEVEL_PROVINCE;
//    public static final String startCode = "440000000000";
//    public static final String startUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/44.html";//startRegion所在的层级，注意startRegion和startLevel一定要匹配才能正常抓取
//    public static String TARGET_LEVEL = LEVEL_COUNTY;

    //抓取东莞市所有街道
//    public static final String startRegion = "东莞市";
//    public static final String startLevel = LEVEL_CITY;
//    public static final String startCode = "441900000000";
//    public static final String startUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/44/4419.html";
//    public static String TARGET_LEVEL = LEVEL_COUNTY;

    //抓取广州天河区到居委会的数据
    public static final String startRegion = "天河区";
    public static final String startLevel = LEVEL_COUNTY;
    public static final String startCode = "440106000000";
    public static final String startUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/44/01/440106.html";
    public static String TARGET_LEVEL = LEVEL_VILLAGE;


    //**************************以上值请根据实际情况修改*************************************

    public static void main(String[] args) throws Exception {
        System.out.println("开始抓取,请耐心等待！！！");
        System.out.println("抓取范围：" + startRegion + " ,抓取层级（" + levelMap + "）：" + levelMap.get(TARGET_LEVEL));
        long starttime = System.currentTimeMillis();
        failedItems.clear();

        Region region = new Region(startCode, startRegion);

        if (LEVEL_CHINA.equals(startLevel)) {
            parseProvinceLevel(startUrl, region);
        } else if (LEVEL_PROVINCE.equals(startLevel)) {
            parseCityLevel(startUrl, region);
        } else if (LEVEL_CITY.equals(startLevel)) {
            if (citysWithNextLevelNotCounty.contains(startRegion)) {
                parseTownLevel(startUrl, region);
            } else {
                parseCountyLevel(startUrl, region);
            }
        } else if (LEVEL_COUNTY.equals(startLevel)) {
            parseTownLevel(startUrl, region);
        } else if (LEVEL_TOWN.equals(startLevel)) {
            parseVillageLevel(startUrl, region);
        }

        //写到文件里
        saveJson(resultStorePath + "/" + startRegion + ".json", region);
        saveJson(errorLogStorePath, failedItems);
        long endtime = System.currentTimeMillis();
        System.out.println("抓取完毕！！！耗时：" + (endtime - starttime) / 1000 / 60 + "min");
        if (!failedItems.isEmpty()) {
            System.out.println("失败" + failedItems.size() + "条,抓取过程中失败的url请查看error.log!");
        }
    }

    /**
     * 解析Province级别
     *
     * @param url 例如：http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html
     */
    private static void parseProvinceLevel(String url, Region father) {
        if (url == null) {
            System.out.println("parseProvinceLevel url==null");
            return;
        }
        Document doc = null;
        try {
            doc = getDocument(url);
        } catch (IOException e) {
            e.printStackTrace();
            failedItems.add(new FailedInfo(url, father));
            return;
        }
        Elements provincetr = doc.getElementsByClass(LEVEL_PROVINCE);
        for (Element e : provincetr) {
            Elements a = e.getElementsByTag("a");
            for (Element ea : a) {
                String nextUrl = null;
                String code = null;
                String name = null;

                nextUrl = ea.attr("abs:href");//拿到绝对路径
                String[] arr = nextUrl.split("/");
                code = arr[arr.length - 1].split("\\.")[0] + "0000000000";
                name = ea.text();//获得名称

                if (father.childs == null) {
                    father.childs = new ArrayList<>();
                }
                Region child = new Region(code, name);
                father.childs.add(child);
                System.out.println(space(levelMap.get(LEVEL_PROVINCE)) + name);
                if (isNeedNext(LEVEL_CITY)) {
                    parseCityLevel(nextUrl, child);
                }

            }
        }
    }

    /**
     * 解析City级别
     *
     * @param url 例如：http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/11.html
     */
    private static void parseCityLevel(String url, Region father) {
        if (url == null) {
            System.out.println("parseCityLevel url==null");
            return;
        }
        Document doc = null;
        try {
            doc = getDocument(url);
        } catch (IOException e) {
            e.printStackTrace();
            failedItems.add(new FailedInfo(url, father));
            return;
        }
        Elements citytr = doc.getElementsByClass(LEVEL_CITY);
        for (Element e : citytr) {
            String nextUrl = null;
            String code = null;
            String name = null;

            Elements a = e.getElementsByTag("a");
            if (a.isEmpty()) {//表示没有下一级了
                nextUrl = null;
                Elements tds = e.getElementsByTag("td");
                code = tds.get(0).text();
                name = tds.get(1).text();
            } else {//<td><a href='44/4401.html'>440100000000</a></td><td><a href='44/4401.html'>广州市</a></td>
                nextUrl = a.get(0).attr("abs:href");
                code = a.get(0).text();
                name = a.get(1).text();//获得名称
            }

            if (father.childs == null) {
                father.childs = new ArrayList<>();
            }
            Region child = new Region(code, name);
            father.childs.add(child);
            System.out.println(space(levelMap.get(LEVEL_CITY)) + name);
            if (citysWithNextLevelNotCounty.contains(name)) {
                if (isNeedNext(LEVEL_TOWN)) {
                    parseTownLevel(nextUrl, child);
                }
            } else {
                if (isNeedNext(LEVEL_COUNTY)) {
                    parseCountyLevel(nextUrl, child);
                }
            }
        }
    }

    /**
     * 解析County级别
     *
     * @param url
     * @param father
     */
    private static void parseCountyLevel(String url, Region father) {
        if (url == null) {
            System.out.println("parseCountyLevel url==null");
            return;
        }
        Document doc = null;
        try {
            doc = getDocument(url);
        } catch (IOException e) {
            e.printStackTrace();
            failedItems.add(new FailedInfo(url, father));
            return;
        }
        Elements countytr = doc.getElementsByClass(LEVEL_COUNTY);
        for (Element e : countytr) {
            String nextUrl = null;
            String code = null;
            String name = null;

            Elements a = e.getElementsByTag("a");
            if (a.isEmpty()) {//<td>440101000000</td><td>市辖区</td>
                nextUrl = null;
                Elements tds = e.getElementsByTag("td");
                code = tds.get(0).text();
                name = tds.get(1).text();
            } else {//<td><a href='01/440103.html'>440103000000</a></td><td><a href='01/440103.html'>荔湾区</a></td>
                nextUrl = a.get(0).attr("abs:href");
                code = a.get(0).text();
                name = a.get(1).text();//获得名称
            }

            if (father.childs == null) {
                father.childs = new ArrayList<>();
            }
            Region child = new Region(code, name);
            father.childs.add(child);
            System.out.println(space(levelMap.get(LEVEL_COUNTY)) + name);

            if (isNeedNext(LEVEL_TOWN)) {
                parseTownLevel(nextUrl, child);
            }

        }
    }

    /**
     * 解析Town级别
     *
     * @param url
     * @param father
     */
    private static void parseTownLevel(String url, Region father) {
        if (url == null) {
            System.out.println("parseTownLevel url==null");
            return;
        }
        Document doc = null;
        try {
            doc = getDocument(url);
        } catch (IOException e) {
            e.printStackTrace();
            failedItems.add(new FailedInfo(url, father));
            return;
        }
        Elements towntr = doc.getElementsByClass(LEVEL_TOWN);
        for (Element e : towntr) {
            String nextUrl = null;
            String code = null;
            String name = null;

            Elements a = e.getElementsByTag("a");
            if (a.isEmpty()) {//<td>440101000000</td><td>市辖区</td>
                nextUrl = null;
                Elements tds = e.getElementsByTag("td");
                code = tds.get(0).text();
                name = tds.get(1).text();
            } else {//<td><a href='01/440103.html'>440103000000</a></td><td><a href='01/440103.html'>荔湾区</a></td>
                nextUrl = a.get(0).attr("abs:href");
                code = a.get(0).text();
                name = a.get(1).text();//获得名称
            }

            if (father.childs == null) {
                father.childs = new ArrayList<>();
            }
            Region child = new Region(code, name);
            father.childs.add(child);
            System.out.println(space(levelMap.get(LEVEL_TOWN)) + name);
            if (isNeedNext(LEVEL_VILLAGE)) {
                parseVillageLevel(nextUrl, child);
            }

        }
    }

    /**
     * 解析Village级别
     *
     * @param url
     * @param father
     */
    private static void parseVillageLevel(String url, Region father) {
        if (url == null) {
            System.out.println("parseVillageLevel url=null");
            return;
        }
        Document doc = null;
        try {
            doc = getDocument(url);
        } catch (IOException e) {
            e.printStackTrace();
            failedItems.add(new FailedInfo(url, father));
            return;
        }
        Elements villagetr = doc.getElementsByClass(LEVEL_VILLAGE);
        for (Element e : villagetr) {
            String nextUrl = null;
            String code = null;
            String name = null;

            Elements tds = e.getElementsByTag("td");
            if (tds.size() == 3) {
                code = tds.get(0).text();
                name = tds.get(2).text();
            } else {
                code = tds.get(0).text();
                name = tds.get(1).text();
            }

            if (father.childs == null) {
                father.childs = new ArrayList<>();
            }
            father.childs.add(new Region(code, name));
            System.out.println(space(levelMap.get(LEVEL_VILLAGE)) + name);

        }
    }

    /**
     * 判断是否需要往下继续解析
     *
     * @param level
     * @return
     */
    private static boolean isNeedNext(String level) {
        if (levelMap.get(TARGET_LEVEL) >= (levelMap.get(level))) {
            return true;
        }
        return false;
    }

    /**
     * 抓取网页内容可能会遇到反爬（如遇到“请开启JavaScript并刷新该页.“的错误提示）
     * 因此需要增加一些手段进行规避（比如改变ua,延时请求等等），如果还是遇到反爬请自行百度解决
     *
     * @param url
     * @return
     * @throws IOException
     */
    private static Document getDocument(String url) throws IOException {
        Random random = new Random();
        int ms = random.nextInt(100);
        sleep(ms);//随机延时

        try {
            return Jsoup.connect(url)
                    //ua可以从浏览器拷过来
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36 Edg/91.0.864.48")
                    .post();//这里使用post来规避反爬，使用get的话容易遇到反爬
        } catch (IOException e) {//第一次遇到异常则延时后再次尝试
            e.printStackTrace();
            sleep(1000);
            return Jsoup.connect(url)
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36 Edg/91.0.864.48")
                    .post();
        }
    }

    private static void sleep(int ms) {
        try {
            Thread.sleep(ms);
        } catch (InterruptedException ignore) {
        }
    }

    /**
     * 将对象序列化为json后保存到指定路径
     *
     * @param savePath
     * @param region
     * @throws IOException
     */
    private static void saveJson(String savePath, Object region) throws IOException {
        File file = new File(savePath);
        if (!file.getParentFile().exists()) {
            file.getParentFile().mkdirs();
        }
        FileWriter fw = new FileWriter(file);
        BufferedWriter bw = new BufferedWriter(fw);
        bw.write(new Gson().toJson(region));
        bw.flush();
        bw.close();
    }

    private static String space(int level) {
        if (level > 5) {
            return "";
        }
        return "      ".substring(0, level);
    }


    //区划数据保存实体类定义
    public static class Region {
        public String code; //编码
        public String name;  //名称
        public List<Region> childs;//下一级

        public Region(String code, String name) {
            this.code = code;
            this.name = name;
        }

        @Override
        public String toString() {
            return "Region{" +
                    "code='" + code + '\'' +
                    ", name='" + name + '\'' +
                    ", childs=" + childs +
                    '}';
        }
    }

    public static class FailedInfo {
        public String url;
        public Region region;

        public FailedInfo(String url, Region region) {
            this.url = url;
            this.region = region;
        }

        @Override
        public String toString() {
            return "FailedInfo{" +
                    "url='" + url + '\'' +
                    ", region=" + region +
                    '}';
        }
    }


}
